{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import math\n",
    "import os\n",
    "import pickle\n",
    "import random\n",
    "import signal\n",
    "import warnings\n",
    "from collections import defaultdict\n",
    "from random import shuffle\n",
    "\n",
    "import multitasking\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from annoy import AnnoyIndex\n",
    "from gensim.models import Word2Vec\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "max_threads = multitasking.config['CPU_CORES']\n",
    "multitasking.set_max_threads(max_threads)\n",
    "multitasking.set_engine('process')\n",
    "signal.signal(signal.SIGINT, multitasking.killall)\n",
    "\n",
    "seed = 2020\n",
    "random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word2vec(df_, f1, f2, model_path):\n",
    "    df = df_.copy()\n",
    "    tmp = df.groupby(f1, as_index=False)[f2].agg(\n",
    "        {'{}_{}_list'.format(f1, f2): list})\n",
    "\n",
    "    sentences = tmp['{}_{}_list'.format(f1, f2)].values.tolist()\n",
    "    del tmp['{}_{}_list'.format(f1, f2)]\n",
    "\n",
    "    words = []\n",
    "    for i in range(len(sentences)):\n",
    "        x = [str(x) for x in sentences[i]]\n",
    "        sentences[i] = x\n",
    "        words += x\n",
    "\n",
    "    if os.path.exists(f'{model_path}/w2v.m'):\n",
    "        model = Word2Vec.load(f'{model_path}/w2v.m')\n",
    "    else:\n",
    "        model = Word2Vec(sentences=sentences,\n",
    "                         size=256,\n",
    "                         window=3,\n",
    "                         min_count=1,\n",
    "                         sg=1,\n",
    "                         hs=0,\n",
    "                         seed=seed,\n",
    "                         negative=5,\n",
    "                         workers=10,\n",
    "                         iter=1)\n",
    "        model.save(f'{model_path}/w2v.m')\n",
    "\n",
    "    article_vec_map = {}\n",
    "    for word in set(words):\n",
    "        if word in model:\n",
    "            article_vec_map[int(word)] = model[word]\n",
    "\n",
    "    return article_vec_map\n",
    "\n",
    "\n",
    "#@multitasking.task\n",
    "def recall(df_query, article_vec_map, article_index, user_item_dict,\n",
    "           worker_id):\n",
    "    data_list = []\n",
    "\n",
    "    for user_id, item_id in tqdm(df_query.values):\n",
    "        rank = defaultdict(int)\n",
    "\n",
    "        interacted_items = user_item_dict[user_id]\n",
    "        interacted_items = interacted_items[-1:]\n",
    "\n",
    "        for item in interacted_items:\n",
    "            article_vec = article_vec_map[item]\n",
    "\n",
    "            item_ids, distances = article_index.get_nns_by_vector(\n",
    "                article_vec, 100, include_distances=True)\n",
    "            sim_scores = [2 - distance for distance in distances]\n",
    "\n",
    "            for relate_item, wij in zip(item_ids, sim_scores):\n",
    "                if relate_item not in interacted_items:\n",
    "                    rank.setdefault(relate_item, 0)\n",
    "                    rank[relate_item] += wij\n",
    "\n",
    "        sim_items = sorted(rank.items(), key=lambda d: d[1], reverse=True)[:50]\n",
    "        item_ids = [item[0] for item in sim_items]\n",
    "        item_sim_scores = [item[1] for item in sim_items]\n",
    "\n",
    "        df_temp = pd.DataFrame()\n",
    "        df_temp['article_id'] = item_ids\n",
    "        df_temp['sim_score'] = item_sim_scores\n",
    "        df_temp['user_id'] = user_id\n",
    "\n",
    "        if item_id == -1:\n",
    "            df_temp['label'] = np.nan\n",
    "        else:\n",
    "            df_temp['label'] = 0\n",
    "            df_temp.loc[df_temp['article_id'] == item_id, 'label'] = 1\n",
    "\n",
    "        df_temp = df_temp[['user_id', 'article_id', 'sim_score', 'label']]\n",
    "        df_temp['user_id'] = df_temp['user_id'].astype('int')\n",
    "        df_temp['article_id'] = df_temp['article_id'].astype('int')\n",
    "\n",
    "        data_list.append(df_temp)\n",
    "\n",
    "    df_data = pd.concat(data_list, sort=False)\n",
    "\n",
    "    os.makedirs('../user_data/tmp/w2v', exist_ok=True)\n",
    "    df_data.to_pickle('../user_data/tmp/w2v/{}.pkl'.format(worker_id))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_click = pd.read_pickle('./user_data/data/offline/click.pkl')\n",
    "df_query = pd.read_pickle('./user_data/data/offline/query.pkl')\n",
    "\n",
    "os.makedirs('../user_data/data/offline', exist_ok=True)\n",
    "os.makedirs('../user_data/model/offline', exist_ok=True)\n",
    "\n",
    "w2v_file = '../user_data/data/offline/article_w2v.pkl'\n",
    "model_path = '../user_data/model/offline'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "article_vec_map = word2vec(df_click, 'user_id', 'click_article_id',\n",
    "                               model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(w2v_file, 'wb')\n",
    "pickle.dump(article_vec_map, f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "    # 将 embedding 建立索引\n",
    "article_index = AnnoyIndex(256, 'angular')\n",
    "article_index.set_seed(2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████| 34739/34739 [00:01<00:00, 21975.55it/s]\n"
     ]
    }
   ],
   "source": [
    "for article_id, emb in tqdm(article_vec_map.items()):\n",
    "    article_index.add_item(article_id, emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "article_index.build(100)\n",
    "user_item_ = df_click.groupby('user_id')['click_article_id'].agg(lambda x: list(x)).reset_index()\n",
    "user_item_dict = dict(zip(user_item_['user_id'], user_item_['click_article_id']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 召回\n",
    "n_split = max_threads\n",
    "all_users = df_query['user_id'].unique()\n",
    "shuffle(all_users)\n",
    "total = len(all_users)\n",
    "n_len = total // n_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 11308/11308 [01:47<00:00, 105.44it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 11308/11308 [01:46<00:00, 106.67it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 11308/11308 [01:46<00:00, 106.61it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 11308/11308 [01:45<00:00, 106.73it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 11308/11308 [01:45<00:00, 106.75it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 11308/11308 [01:45<00:00, 106.75it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 11308/11308 [01:45<00:00, 106.88it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 11308/11308 [01:45<00:00, 106.85it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 106.67it/s]\n"
     ]
    }
   ],
   "source": [
    "# 清空临时文件夹\n",
    "for path, _, file_list in os.walk('../tmp/w2v'):\n",
    "    for file_name in file_list:\n",
    "        os.remove(os.path.join(path, file_name))\n",
    "\n",
    "for i in range(0, total, n_len):\n",
    "    part_users = all_users[i:i + n_len]\n",
    "    df_temp = df_query[df_query['user_id'].isin(part_users)]\n",
    "    recall(df_temp, article_vec_map, article_index, user_item_dict, i)\n",
    "\n",
    "multitasking.wait_for_tasks()\n",
    "\n",
    "\n",
    "df_data = pd.DataFrame()\n",
    "for path, _, file_list in os.walk('../user_data/tmp/w2v'):\n",
    "    for file_name in file_list:\n",
    "        df_temp = pd.read_pickle(os.path.join(path, file_name))\n",
    "        df_data = df_data.append(df_temp)\n",
    "\n",
    "# 必须加，对其进行排序\n",
    "df_data = df_data.sort_values(['user_id', 'sim_score'],\n",
    "                                  ascending=[True,\n",
    "                                             False]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data.to_pickle('./user_data/data/offline/recall_w2v.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf1.14]",
   "language": "python",
   "name": "conda-env-tf1.14-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
